import requests
from requests.exceptions import RequestException
from pyquery import PyQuery as pq
import time, json

# 抓取页面信息
def getPage(url):
    try:
        headers = {
            'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36',
        }
        res = requests.get(url, headers=headers)
        if res.status_code == 200:
            return res.text
        else:
            return None
    except RequestException:
        return None

# 解析网页信息

def parsePage(content, n):
    html = pq(content)
    items = html('tr.item')
    for item in items.items():
        n += 1
        yield {
            'index':n,
            'img':item.find('td a img').attr('src'),
            'title':item.find('div.pl2 a').text(),
            'author':item.find('p.pl').text(),
            'score':item.find('div[class="star clearfix"] span:eq(1)').text(),
        }

# 解析内容打印或写入文件
def writeFile(content):
    with open('./topbook.txt', 'a', encoding='utf-8') as f:
        f.write(json.dumps(content, ensure_ascii=False)+'\n')

# 主函数
def main(offset):
    url = 'https://book.douban.com/top250?start=%d' % (offset)
    html = getPage(url)
    if html:
        for item in parsePage(html, offset):
            writeFile(item)
            # print(item)
    print('成功写入%d条数据' % (offset+25))

if __name__ == '__main__':
    for offset in range(10):
        main(offset=offset*25)
        time.sleep(1)